#include "model/qwen2_hub.h"
#include <gtest/gtest.h>
#include <string>

using namespace pa;

TEST(Qwen2HubTest, is_valid_check) {
    std::string hub = "data/Qwen2-0.5B";
    Qwen2Hub hub_model(ModelBase::MODEL_QWEN2_HUB, hub);
    bool flag = hub_model.is_valid();
    EXPECT_TRUE(flag);
}

TEST(Qwen2HubTest, load_test) {
    std::string hub = "data/Qwen2-0.5B";
    Qwen2Hub hub_model(ModelBase::MODEL_QWEN2_HUB, hub);
    bool flag = hub_model.load();
    EXPECT_TRUE(flag);
    EXPECT_EQ(hub_model.token_size(), 151643);
    std::string pattern = "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+";
    EXPECT_EQ(hub_model.pre_token_pattern(), pattern);
}